{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.3 模型评估方法 - ROC曲线与KS曲线"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.案例实战 - 股票客户流失预警模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 1 1 1 0 0 1 0 1 0 0 0 0 1 0 0 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 1 0 0 0 1 0 0 0 1]\n",
      "0.7977288857345636\n",
      "[[0.82041491 0.17958509]\n",
      " [0.84029613 0.15970387]\n",
      " [0.79819342 0.20180658]\n",
      " [0.62989192 0.37010808]\n",
      " [0.61636611 0.38363389]]\n"
     ]
    }
   ],
   "source": [
    "# 1.读取数据\n",
    "import pandas as pd\n",
    "df = pd.read_excel('股票客户流失.xlsx')\n",
    "\n",
    "# 2.划分特征变量和目标变量\n",
    "X = df.drop(columns='是否流失') \n",
    "y = df['是否流失']\n",
    "\n",
    "# 3.划分训练集和测试集\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)\n",
    "\n",
    "# 4.模型搭建\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "model = LogisticRegression()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# 5.模型使用1 - 预测数据结果\n",
    "y_pred = model.predict(X_test)\n",
    "print(y_pred[0:100])  # 打印预测内容的前100个看看\n",
    "\n",
    "# 查看全部的预测准确度\n",
    "from sklearn.metrics import accuracy_score\n",
    "score = accuracy_score(y_pred, y_test)\n",
    "print(score)  # 打印整体的预测准确度\n",
    "\n",
    "# 6.模型使用2 - 预测概率\n",
    "y_pred_proba = model.predict_proba(X_test)  \n",
    "print(y_pred_proba[0:5])  # 打印前5个客户的分类概率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.3 模型评估方法 - ROC曲线与KS曲线"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**4.3.1 分类模型的评估方法 - ROC曲线**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "补充知识点：混淆矩阵的Python实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[968  93]\n",
      " [192 156]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "m = confusion_matrix(y_test, y_pred)  # 传入预测值和真实值\n",
    "print(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0（预测不流失）</th>\n",
       "      <th>1（预测流失）</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0（实际不流失）</th>\n",
       "      <td>968</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1（实际流失）</th>\n",
       "      <td>192</td>\n",
       "      <td>156</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0（预测不流失）  1（预测流失）\n",
       "0（实际不流失）       968       93\n",
       "1（实际流失）        192      156"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.DataFrame(m, index=['0（实际不流失）', '1（实际流失）'], columns=['0（预测不流失）', '1（预测流失）'])\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.91      0.87      1061\n",
      "           1       0.63      0.45      0.52       348\n",
      "\n",
      "    accuracy                           0.80      1409\n",
      "   macro avg       0.73      0.68      0.70      1409\n",
      "weighted avg       0.78      0.80      0.79      1409\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, y_pred))  # 传入预测值和真实值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**4.3.2 案例实战 - 评估股票客户流失预警模型**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.17958509, 0.15970387, 0.20180658, ..., 0.04220544, 0.09782449,\n",
       "       0.63586739])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_proba[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.计算ROC曲线需要的假警报率（fpr）、命中率（tpr）及阈值（thres）\n",
    "from sklearn.metrics import roc_curve\n",
    "fpr, tpr, thres = roc_curve(y_test, y_pred_proba[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 感兴趣的读者可以查看下roc_curve()函数返回的内容\n",
    "# print(roc_curve(y_test, y_pred_proba[:,1]))\n",
    "# type(roc_curve(y_test, y_pred_proba[:,1]))\n",
    "# len(roc_curve(y_test, y_pred_proba[:,1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>阈值</th>\n",
       "      <th>假警报率</th>\n",
       "      <th>命中率</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.930369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.930369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.867342</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.034483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.864187</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.034483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.857303</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.040230</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         阈值      假警报率       命中率\n",
       "0  1.930369  0.000000  0.000000\n",
       "1  0.930369  0.000000  0.002874\n",
       "2  0.867342  0.000000  0.034483\n",
       "3  0.864187  0.001885  0.034483\n",
       "4  0.857303  0.001885  0.040230"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2.查看假警报率（fpr）、命中率（tpr）及阈值（thres）\n",
    "a = pd.DataFrame()  # 创建一个空DataFrame \n",
    "a['阈值'] = list(thres)\n",
    "a['假警报率'] = list(fpr)\n",
    "a['命中率'] = list(tpr)\n",
    "a.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAESCAYAAAAVLtXjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXhV5b328e8vIYxhlACihElQEUUgVUCEOE9V3zpUrbWeOnC0LT3ntLXHVmvtqZ2o7Wmrr63Uofqqtc7WEbSKojKrKIMMMoNhCkMgkPH3/rF3QoCEhLDXXnvvdX+ui8s9PNn7XgburKy91vOYuyMiItGSFXYAERFJPpW/iEgEqfxFRCJI5S8iEkEqfxGRCFL5iySQmeWGnUGkKVT+IofIzD43s8Fmlg2UmFmrBsaNMrMRde6faWYXx2/r36Iklf7CSaSY2Z1mtsPMNpjZWjP7fvzxG81snZkVmdm364wfambz48/9ooGX3Q1sd/cqoNrdyxoY93Mgr87944BCM7seuOfQt06k6VT+EkX3uns34BTgv81sCPAboBAYBfzczI41sxbAM8CPgT7AGWZ2dj2v50B1ndv7MbMxwDpglpldFn94J7EfHH8DZsbHiCRFi7ADiITF3VeY2XTgXGCSuy8GMLPXgQuA2cAud38x/vgLwBnAZDP7iNi/n21AX+A5MysHss3svfhbnAK0A3KBCcDlwA+BNmZWBZwDnAScHH+d2cC7gW+4CCp/iTAzywcKgPbAjDpPrSK2p78bWFTn8YeJlTlABfBf7j7FzOYBl7j7GjOrdPfR8d8aKuJ/TgbygRfiX38acGv8tUvd/dp4nsFm1svdVweywSJ16LCPRNF3zGwDsAT4LbAMqHucvhxoA3QCdtQ86O7r3X1Z/G41TePAy8ARxPbqvwVsBJ4AqoBTzez1+G8gd8XHiQROe/4SRfcC/wOsBl4B+gGt6zzfCiglttdee+aOmY0Fern7Y/GH7jGzAx32AcDd3cy+TqzYPyf2W8WNwCxiO2BfBb4MlLj79MRuqkj9tOcvkeTupcBDxPbElxEr5Bq9geXAUmI/GGqcCpxQ5/54dx8dH3tJ/HZV/L+FNYPMLIc9e/W/Bo5x9xvc/X5gBXB9/PmSxG2hyIGp/CXK7gWuAd4Czjazo82sH3A2sd8I3gD6mtkZ8Yu3LgfePtg3cfcKd+8DnA78HjiyztOLiRX/OcA7h7AtIgdFh30kstx9pZm9S+wD2VuAKcR2iG5z90UAZnYBMBHoBjzg7q/Fv9xo4mEfMzsWmERsz34q8L6Z9QHujr/fTGLlf6qZ5bv7nUFts0gN02IuIgfPzD4G/tPdpzTwfM3ZPu3cvTRe6qvizw0n9pvFHe4+0cw6x+8XAIXu/kFSNkIiTeUvEgIza+nu5XXuZwHd3L0oxFgSISp/EZEI0ge+IiIRpPIXEYmgtDjbp2vXrt6nT5+wY4iIpJU5c+Zscve8+p5Li/Lv06cPs2fPDjuGiEhaMbOVDT2nwz4iIhGk8hcRiSCVv4hIBKn8RUQiSOUvIhJBKn8RkQgKpPzNrLuZTT3A8zlm9pKZvW9m1wWRQUREGpbw8/zjMxQ+wp61TuszHpjj7nea2atm9rS7ayELEUlbi9eX8PLcdQl/3YI+XRgzsN7rtA5JEBd5VQFXAC8eYEwhsQWsIbauaQH7LJJhZuOAcQD5+fkJDyki0pB3Fm9kY0nZAcc8PmMlC7/YTk527ABKye5KAMwSm+Wmsf3To/zdfTuAHfj/QDtgbfx2MdC9nteZSGwRDQoKCjT1qIgclG2lFSzecHAHFDaVlPGTF+exaUd544Pjrjppz87p8N6d+fIJPQ/qPcMS1vQOO4A2wDYgN35fROSguDvvL93MY9NXkrXPJ5ivftr8pRF6dGjN3ZcPofdhbQ84Lq99K1rnZDf7fcIUVvnPAUYDzwBDgOkh5RCRNPH/pq1g7dbdez320PvLKa+sBqBv13a0yNpzxOGobrn07tKWb57S96Dep12rbE7s1amxoxdpL/DyN7PTgUHufm+dhx8BXjWzU4FBwIygc4hI4mwrrWDSgiKqqoM9IvvRqi08NXvNXo+1bLFnF7+8spo2Odn85ZrhjA3guHgmC6z83b0w/t+3gLf2eW6lmZ1FbO//DnevCiqHiBy6TTvKWFy05/j5NQ/NDLz467qioBdd27fk2pF96NahddLeN5OFNqWzu68Dngrr/UUEtu2qoGR3Rb3PucNvJy2itLyKNxeur3fM9B+dEWQ8IHYYpn3rnMDfJ2rSYj5/ETk0k+cXMWN58V6P7Syr5MlZq5v09cce3oGB3XP5Wp0zW47u0Z5ObVsmNKckj8pfJEO4Oy9/8gVbSvecpvjLVxeyu6K69n5uqz3/5Curq8kyuHTYkXypb5d6XzMn2zhrUI+9vk4yg76jIimsqtqZubyYssq9PxZbsWkn9035nDYts8mKn5WyfvtuSsv3//isZXYW48b049zBPRh8RMek5JbUp/IXSUE7yirZWFLGr19byKT59R9vBxia34lenWPnoh8fL/abxvane4dWtWO6tGuZ8actysFT+YukmOWbdnLa3VP2euyJG06mdcu9Lybq0DqHo7rlJjGZZBKVv0iKqKp2/j5zFbe/MA+AUwd05dJhR3J0j/Yce3iHkNNJplH5i6SI8X//sHZKgotP7MkfrxwaciLJZCp/kSRyd1ZsLqWias8ZOCW7K/jNa4tY8MV2IHbufI+OupBJgqXyF0mSRUUlXPe3Wazduqve51vnZDHh0hNU/JIUKn+RAG0rreDpOaupqHLeWbyBtVt30Toni1/8n+P3mg2ybatsxgzIIztLZ+VIcqj8RQJy5z/n87cPVuz12IBuubzxvbHhBBKpQ+UvkiCri0v58fOfsqu8iuLScpZt3AnAtwr7M/70AZhRu+qTSNhU/iKHqKrauePFeTw+Y1XtY6P6H8Zh7Vry4/OPZWh+5xDTidRP5S9yCNydY+94vXZBkVvOOZp/G9WHdpoLR1Kc/oaKHKSKqmr++fE6vv/03L0en3XbmeS1b9XAV4mkFpW/yAFs313B0g17LzE97tE5bNpRBsQ+wB07MI9vnXYUXdppemNJHyp/kQP43j8+5s2FG+p97s3vjdXcOpK2VP4i9Zi5vJipSzay8IsSjunRnlvPO2av54fmd6ZjG60uJelL5S+RVl3tXPvwTGYsL6ZlndMwd5RVApBlcOaxvSk8ultYEUUCofKXSNq8o4ylG3awqriUqUs2AXDNiN57jTnj2G6M6t81jHgigVP5S6S4O8/MWcMtz3yy1+OPXncSYwbmhZRKJPlU/hIJKzfv5MlZq3l69praM3W+fMLhfO2kfNq0zGbIkZ1CTiiSXCp/yVg7yyp55dMvqKiq5rbnYwuk1Myb9tJ3RnP8kVrPVqJL5S8ZY2tpOfPWbucfs1ezYN02Po/PrVOjX1473vp+YTjhRFKMyl/SVsnuCraWVgCwtbSCC+99b6/nzz++B61bZPPDc48hy6Brrq6+Famh8pe0tLOskuPvnLzf4/3y2vGbS0+gX9d2HKayF2mQyl/Szl0vL+CB95YDMLB7LuPG9Acgt1ULzh7UnSwtiCLSKJW/pI3PirbzxdbdTF++me4dWnH96L5885S+miNfpBlU/pKynp69mn/OXQeAO7y3dFPtc6cO6Fq7xy8iB0/lLynruQ/X8smarQzs0R53OP6Ijlx8Yk8K+nShb9d2YccTSWuBlL+ZPQgMAl5x97vqeb4z8DjQDZjj7v8eRA5JXx98vol123ZxXM+OPHXTyLDjiGSchB8sNbNLgGx3Hwn0M7MB9Qy7Bnjc3QuA9mZWkOgckr627arga3+dwcrNpRzZpU3YcUQyUhB7/oXAU/Hbk4HRwJJ9xmwGBptZJ6AXsDqAHJImqqudzzfuoMqdfy3cwOvzigD4xsje/Oyi40JOJ5KZgij/dsDa+O1iYFg9Y94DLgC+CyyMj9uLmY0DxgHk5+cHEFNSxdl/eHe/1bLGDMxj3Jh+mOm0TZEgBFH+O4Ca39Vzqf/Q0k+Bm9x9u5l9D/gmMLHuAHefWPNYQUGBB5BTUsCOssra4r/v6mEYcFS3XAZ0bx9uMJEMF0T5zyF2qGc6MARYVM+YzsDxZjYdOBl4M4AckmJmrShm/fbdez324cqtAPzovGM4//jDw4glEklBlP8LwFQz6wmcB1xpZne5++11xvwKeBjoDUwD/h5ADkkhKzbt5PK/TGvw+YI+XZKYRkQSXv7xQzmFwFnABHcvAubuM2YmoE/yMtyTM1cxc3ns45znPop9DHTLOUdz9qDue43r0CaH7h1aJz2fSJQFcp6/u29hzxk/EkFTl2zk1uc+BaBXlzYc3rE1YwfmccOpfWnVIjvkdCKiK3wl4V7+ZB3/HV8mUcsjiqQmlb8k1IertvCdJz4C4AdnD+TUAVoAXSQVqfwloW59NrbHf9VJ+Xzn9Pou7haRVKDyl4T4ZM1Wrn9kNhtLyujQugW/uuT4sCOJyAGo/OWgzV+3jflrt9feX1Vcyr1vL629//gNI8KIJSIHQeUvTbKttIIFX8QK/9bnPmHl5tL9xtx2/rHcOKZfsqOJSDOo/GU/7s6aLbv41WsLqaqOzawxaf76vcZcfGJPfnjuMbX32+Zk07ldy6TmFJHmU/nLfi7/yzRmr9xSe/+YHu05unt7enVpy/Wj+wIw+IgOtG+dE1ZEETlEKn+pNXtFMZfVmYLhD1ecyLmDe9A6RxdliWQalb9QVe3MWbmFbz3+IQD989rx7M2j6NRWh3FEMpXKP+LKK6u55Zm5vPhxbKH0rww9gv+94sSQU4lI0FT+ETf8529QUlYJxKZiOKmvZtcUiQKVf4S9vWhDbfG/PH40g4/oGHIiEUkWlX/EbC0t5+sPzmD7rkpWFcfO1f/TVUNV/CIRo/KPmN9NXsy8tdsZ0a8Lw/I78ZVhRzJWs26KRI7KP0L+MWsVr80rAuB/rziRwzu2aeQrRCRTqfwjZMLri9hRVsl/nDFAxS8ScSr/DFReWc17SzdSXlkNQGW186PnPqVkdyXfGNmb/zprYMgJRSRsKv8M9MaC9Xz7iQ/3e7xjmxwuG35kCIlEJNWo/DPMoqISHp22AoCH/+1L9OgYWxg9J9von5eLmYUXTkRShso/w7w0dx0zlhdz/BEdGdHvMNq01Lw8IrI/lX8G2V1RVbuoykvjR4ecRkRSWVbYASRxHpu+EoDWOfq2isiBac8/A7g7v37tM+5/dxkAU35wWsiJRCTVqfwzwMaSMu5/dxkdWrfgmpG9az/kFRFpiMo/zS1eX8L7SzcB8N/nHcPVJ/cOOZGIpAOVfxpbu3UX3378Q5Zs2AHAYVpDV0SaSOWfpp6cuYpbn/sUgHOP68FPLxqkKRtEpMlU/mnqwfeWA3DdKX357hlHaclFETkoKv809Kd/LWHJhh2cc1x37rhwUNhxRCQNqfzTyPtLN3Hb85+yYnNsEZZrR/YJN5CIpK1ArgYyswfNbJqZ3d7IuPvM7MIgMmSa2SuKufqBGazYXMqo/ofxxI0nM+qormHHEpE0lfA9fzO7BMh295Fm9pCZDXD3JfWMOxXo4e4vJTpDJlmyvoSnZq/mg883A3DVSb341SUnhJxKRNJdEId9CoGn4rcnA6OBvcrfzHKAvwKvmtnF7v7ivi9iZuOAcQD5+fkBxEx9v3ptIfe/E7tqt13LbEYf1VXFLyIJEUT5twPWxm8XA8PqGfMNYAEwARhvZvnufk/dAe4+EZgIUFBQ4AHkTHn3v7OMvPatuPPC47jghMPDjiMiGSSIY/47gJoTznMbeI+hwER3LwIeAzQZTVx1tXPvW0u49qGZAPTq3EbFLyIJF8Se/xxih3qmA0OARfWMWQr0i98uAFYGkCPtvLdkE9c8NAOP/54zNL8TPzz3mHBDiUhGCqL8XwCmmllP4DzgSjO7y93rnvnzIPCQmV0J5ACXBZAjrby5YD03PDobgM5tc3j25lH0y8sNOZWIZKqEl7+7bzezQuAsYEL80M7cfcaUAJcn+r3T2V+nxj7YveeqoVw4pGfIaUQk0wVykZe7b2HPGT/SBDOWFwOo+EUkKbTkUwpYv303ACf37RJyEhGJCpV/yHaVV9Uuv6izekQkWTS3T8h+N3kRD7y3nBZZxnE9O4YdR0QiQuUfkjcXrOen/5zP2q27AHjje2Pp27VdyKlEJCpU/iF5c+F6NpTspqB3Z2497xgVv4gklco/BNXVzvMfraVjm5Y8c/OosOOISASp/JPI3ZmyeCOLi0ooq6ymTXV12JFEJKJU/klSXe2cOuHt2mP8AH/5+vAQE4lIlB2w/M0sGzgTKHf3t+OPGXCpuz+ThHwZY/KCotri//uNI+h9WFt6dtKC6yISjsb2/J8AdgK5ZvYV4HPgBuBfgMq/iaqqnZse+xCASf85hqN7tA85kYhEXWPl38vdR8X39pcD9wGnuvvW4KNlhrLKKh75YAUAndrmMLC7JmsTkfA1Vv6tzWwksSuBi4H3gEFmhrt/EHi6DPD2Zxv55aufAfDgtV8i9nNURCRcjZX/XODGem47oPJvgvKq2Bk9z948kuG9O4ecRkQkprHy/zEwHigF/hifilmaoWOblmFHEBGp1djEbo8C84GtxI73y0HYsrOc7/79IwCydLRHRFJIY3v+Ld39cQAzi/xqWwejqtoZ+vM3au/3OUzTN4hI6mis/PPM7GuAAd3itwFw9ycCTZbm5q6JnRDVJiebj396Flna9ReRFNJY+ZcDA+K3/1HntjSgeGc5f526jD9P+RyAP155Iq1aZIecSkRkb42Vf7G7/ywpSTLEhNc/48lZq2mRZZwzuAdnH9cj7EgiIvtprPxHmNnifR4zwN19YECZ0tbdkxbx5KzVAMy+/Uw6tdUZPiKSmhor/xnuflpSkmSAdxZvBGJz96j4RSSVNXaqp+bvaaKNJWV8unYbpx2dx8j+h4UdR0TkgA5Y/u7+f5MVJN3d+uwnAOR3aRtyEhGRxjW25y9N4O4s/GI7AHdedFzIaUREGqfyT4AFX2xn3bbddGvfShO3iUhaUPknwLy12wD4yZcHhZxERKRpVP4J8PInXwBwYq9OIScREWkalf8hKq+sZuqSTQD00oe9IpImVP6H6IkZKwG4aEjPkJOIiDSdyv8QTVu2GYDxpx8VchIRkaZr7ArfZjGzB4FBwCvuftcBxnUHXnf3oUHkCEp5ZTWPTlvBtl0VTJq/nmN6tGdAdy3KLiLpI+Hlb2aXANnuPtLMHjKzAe6+pIHhdwNtEp0haPdNWcof3oxtUoss49/H9gs5kYjIwQliz78QeCp+ezIwGtiv/M3sdGAnUBRAhkC9+PE6AOb97Bza5mRrrn4RSTtBHPNvB6yN3y4Guu87wMxaAj8Bbm3oRcxsnJnNNrPZGzduDCBm81RXO6XllQDktmqh4heRtBRE+e9gz6Gc3Abe41bgPnff2tCLuPtEdy9w94K8vLwAYjbP+X+ayvrtZVxwwuFhRxERabYgyn8OsUM9AEOAFfWMORP4tplNAU40swcCyJFwz3+0hs+KSgD4wdlHh5xGRKT5gjjm/wIw1cx6AucBV5rZXe5+e80Adx9Tc9vMprj7DQHkSLhbno7N3PnW98fSt6sWZBeR9JXw8nf37WZWCJwFTHD3ImDuAcYXJjpDELbsLKey2gHol5cbchoRkUMTyHn+7r6FPWf8ZIRdFVUA3Hb+sSEnERE5dIGUf6b5dM023l0SO+OoQxv9LxOR9Kcma8Sj01Zwx4vza+93a986vDAiIgmi8j8Ad+eJGasA+O1lJ3DWoO5amF1EMoLK/wDum/I5nxWVMDS/E5cX9Ao7johIwmhWzwas27qL305aBMCES08IOY2ISGKp/BswZsLbAFwy9AjN2CkiGUflX4+7Jy2istrJMvjdV4eEHUdEJOFU/vv4ZM1W7n17KQBP3DgCM03cJiKZR+Vfx6KiEi66930A/nTVUEb0OyzkRCIiwVD51/FZ0XYAvj4iX2vyikhGU/nX8R9PfgzAN0/pG3ISEZFgqfzjPlq1pfZ2f03cJiIZTuUf9+Gq2Loyf756WMhJRESCp/KPmzQvtpTwqP5dQ04iIhI8lT+xOXxmrigGoGPbnJDTiIgET+UPzF4ZO95/TA9dySsi0aDyBz5Zsw2AOy86LuQkIiLJofIHXp/3BQB9DtO6vCISDSp/YNnGnQD06KiFWkQkGlT+QOucbM45rnvYMUREkkblD2RlQbuWWtdGRKJD5S8iEkEqfxGRCFL5A6VlVWFHEBFJqsiX/6YdZWzeWU5puX4AiEh0RL7831iwHoCh+Z1CTiIikjyRL//nP1oLwAUnHB5yEhGR5Il8+c9cHpvQ7YhObUJOIiKSPJEu/8qqagAGH9FBC7WLSKREtvy37Cxn+F1vAnDeYB3yEZFoiWT5uzuX/vkDtu2q4MIhPbl02JFhRxIRSapAyt/MHjSzaWZ2ewPPdzSz18xsspk9b2Ytg8jRkLc+28CyTbHJ3O748iBN6CYikZPw8jezS4Bsdx8J9DOzAfUMuxr4vbufDRQB5yY6x4G8uXADAM/ePIq89q2S+dYiIikhiNnMCoGn4rcnA6OBJXUHuPt9de7mARv2fREzGweMA8jPz09oQDPomtuK4b07J/R1RUTSRRCHfdoBa+O3i4EG50o2s5FAZ3efvu9z7j7R3QvcvSAvLy+AmCIi0RVE+e8Aak6az23oPcysC3APcF0AGRpUWl7JEzNWUVVdncy3FRFJKUGU/xxih3oAhgAr9h0Q/4D3aeBH7r4ygAwNevj9WJx+ebnJfFsRkZQSRPm/AFxjZr8HvgrMN7O79hlzPTAMuM3MppjZFQHkqD9cfDqHR647KVlvKSKSchL+ga+7bzezQuAsYIK7FwFz9xnzZ+DPiX7vxixZX8KSDTvIzjJyW2nlLhGJrkAa0N23sOeMn5TxeXyh9u+fPTDkJCIi4YrYFb4OQOHAbiHnEBEJV6TK/5VPiwDIydYkbiISbZEq/9xW2QAc1U1n+ohItEWq/AG6tW+l6ZtFJPIiV/4iIhKx8p+xrJhq97BjiIiELlInu2/dVUHxzvKwY4iIhC4ye/4lu2PF/9UCLdwiIhKZ8l+7dRcAHdvkhJxERCR8kSn/GsPyNYe/iEhkyv/uSYsAyMrSaZ4iIpEp/5LdlQCcclTXkJOIiIQvMuUPcHLfLprNU0SECJW/zu4XEdkjEuW/o6ySmcuLqajS0o0iIhCR8n9vySYAjuzcNuQkIiKpIRLlX7PH/63T+oecREQkNUSi/F/55AsAWrfIDjmJiEhqiET511zV26dru5CTiIikhowv/62l5fxj9mq65rYMO4qISMrI+PJft3U3AMN7a1oHEZEaGV/+Nb4yVLN5iojUiEz5i4jIHhlf/rNWFIcdQUQk5WR8+b/w8VoAhvXuFHISEZHUkdHlv6FkNx+t2gpAXm6rkNOIiKSOjC7/Wcu3AHDll3phpnn8RURqZHT517hudN+wI4iIpJRIlL+IiOxN5S8iEkGBlL+ZPWhm08zs9kMZIyIiwUh4+ZvZJUC2u48E+pnZgOaMERGR4ASx518IPBW/PRkY3cwxIiISkCDKvx2wNn67GOjenDFmNs7MZpvZ7I0bNzYrSI+OrTn/+B5atF1EZB9BtOIOoE38di71/4BpdIy7TwQmAhQUFDRr/fXhvTszvPfw5nypiEhGC2LPfw57DuMMAVY0c4yIiAQkiD3/F4CpZtYTOA+40szucvfbDzBmRAA5RESkAQnf83f37cQ+0J0OnObuc/cp/vrGbEt0DhERaVggn4S6+xb2nM3T7DEiIhIMXeErIhJBKn8RkQhS+YuIRJDKX0Qkgsy9WddPJZWZbQRWNvPLuwKbEhgnHWibo0HbHA2Hss293T2vvifSovwPhZnNdveCsHMkk7Y5GrTN0RDUNuuwj4hIBKn8RUQiKArlPzHsACHQNkeDtjkaAtnmjD/mLyIi+4vCnr+IiOxD5S8iEkEZU/5RXDS+se0xs45m9pqZTTaz582sZbIzJlpTv4dm1t3MPkpWriAdxDbfZ2YXJitXkJrwd7uzmb0aX+3v/mTnC0L87+zUAzyfY2Yvmdn7Znbdob5fRpR/FBeNb+L2XA383t3PBoqAc5OZMdEO8nt4N3tWi0tbTd1mMzsV6OHuLyU1YACauM3XAI/Hz39vb2Zpfe6/mXUGHiG2xG1DxgNz3P0U4DIza38o75kR5U80F40vpJHtcff73P2N+N08YENyogWmkCZ8D83sdGAnsR946a6QRrbZzHKAvwIrzOzi5EULTCGNf583A4PNrBPQC1idnGiBqQKuALYfYEwhe/6/vAsc0g+8TCn/hCwan2aavD1mNhLo7O7TkxEsQI1uc/zQ1k+AW5OYK0hN+T5/A1gATABOMrPxScoWlKZs83tAb+C7wML4uLTl7tubsKhVQjssU8o/IYvGp5kmbY+ZdQHuAQ75GGEKaMo23wrc5+5bk5YqWE3Z5qHARHcvAh4DTktStqA0ZZt/Ctzk7v8DfAZ8M0nZwpTQDkv3AqwRxUXjG92e+F7w08CP3L25E+OlkqZ8D88Evm1mU4ATzeyB5EQLTFO2eSnQL367gOZPgpgqmrLNnYHjzSwbOBmIwgVLie0wd0/7P0AHYC7we2K/Ag4B7mpkTMewcydhm28GtgBT4n+uCDt30Nu8z/gpYWdO0ve5PbEf8u8C04Ajws6dhG0+CZhPbG/4DSA37NwJ2vYp8f+eDnxnn+d6x7f5j8AsYh+KN/u9MuYK3/in5WcB73rs199mjUknmbY9TaFt1jZHmZn1JLb3P8kb/4zgwK+VKeUvIiJNlynH/EVE5CCo/EVEIqhF2AFE0oGZ3UnsIpz18YcGAxvjfyqIXXH6S2IfTu4G1gBfc/eKpIcVaQLt+Ys03S/cvdDdC4F74/fHAA8Tu/QeYLzHpiXYQey0U5GUpPIXOXSdgV01d8zMiF2EUx5aIpFGqPxFmu42M5tiZvfVuf8uMILYudcQu5p6BbHDQ28lP6JI0+iYv0jT/cLdH4PazwBq78cfg9jhn9FAmes8aklh2vMXSbz7gevjUw+IpHWS5DwAAABDSURBVCSVv0iCufsWYod8Lg07i0hDdIWviEgEac9fRCSCVP4iIhGk8hcRiSCVv4hIBKn8RUQiSOUvIhJBKn8RkQj6/+fDKW6jWnagAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 3.绘制ROC曲线\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['font.sans-serif'] = ['SimHei']  # 设置中文\n",
    "plt.plot(fpr, tpr)  # 通过plot()函数绘制折线图\n",
    "plt.title('ROC曲线')  # 添加标题，注意如果要写中文，需要在之前添加一行代码：plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "plt.xlabel('FPR')  # 添加X轴标签\n",
    "plt.ylabel('TPR')  # 添加Y轴标\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8103854528908967"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 4.求出模型的AUC值\n",
    "from sklearn.metrics import roc_auc_score\n",
    "score = roc_auc_score(y_test, y_pred_proba[:,1])\n",
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**补充知识点：对阈值取值的理解**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9303686064600186"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(y_pred_proba[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>分类为0概率</th>\n",
       "      <th>分类为1概率</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>326</th>\n",
       "      <td>0.069631</td>\n",
       "      <td>0.930369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>366</th>\n",
       "      <td>0.085373</td>\n",
       "      <td>0.914627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>662</th>\n",
       "      <td>0.092923</td>\n",
       "      <td>0.907077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1154</th>\n",
       "      <td>0.105118</td>\n",
       "      <td>0.894882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1036</th>\n",
       "      <td>0.105906</td>\n",
       "      <td>0.894094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1093</th>\n",
       "      <td>0.111303</td>\n",
       "      <td>0.888697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164</th>\n",
       "      <td>0.115550</td>\n",
       "      <td>0.884450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>891</th>\n",
       "      <td>0.116594</td>\n",
       "      <td>0.883406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>437</th>\n",
       "      <td>0.123060</td>\n",
       "      <td>0.876940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1153</th>\n",
       "      <td>0.127293</td>\n",
       "      <td>0.872707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>749</th>\n",
       "      <td>0.129633</td>\n",
       "      <td>0.870367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.132658</td>\n",
       "      <td>0.867342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>681</th>\n",
       "      <td>0.133410</td>\n",
       "      <td>0.866590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1327</th>\n",
       "      <td>0.135813</td>\n",
       "      <td>0.864187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>0.136599</td>\n",
       "      <td>0.863401</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        分类为0概率    分类为1概率\n",
       "326   0.069631  0.930369\n",
       "366   0.085373  0.914627\n",
       "662   0.092923  0.907077\n",
       "1154  0.105118  0.894882\n",
       "1036  0.105906  0.894094\n",
       "1093  0.111303  0.888697\n",
       "1164  0.115550  0.884450\n",
       "891   0.116594  0.883406\n",
       "437   0.123060  0.876940\n",
       "1153  0.127293  0.872707\n",
       "749   0.129633  0.870367\n",
       "49    0.132658  0.867342\n",
       "681   0.133410  0.866590\n",
       "1327  0.135813  0.864187\n",
       "264   0.136599  0.863401"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.DataFrame(y_pred_proba, columns=['分类为0概率', '分类为1概率'])\n",
    "a = a.sort_values('分类为1概率', ascending=False)\n",
    "a.head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 补充知识点：KS曲线绘制"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "fpr, tpr, thres = roc_curve(y_test, y_pred_proba[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>阈值</th>\n",
       "      <th>假警报率</th>\n",
       "      <th>命中率</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.930369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.930369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.867342</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.034483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.864187</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.034483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.857303</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.040230</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         阈值      假警报率       命中率\n",
       "0  1.930369  0.000000  0.000000\n",
       "1  0.930369  0.000000  0.002874\n",
       "2  0.867342  0.000000  0.034483\n",
       "3  0.864187  0.001885  0.034483\n",
       "4  0.857303  0.001885  0.040230"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.DataFrame()  # 创建一个空DataFrame \n",
    "a['阈值'] = list(thres)\n",
    "a['假警报率'] = list(fpr)\n",
    "a['命中率'] = list(tpr)\n",
    "a.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAECCAYAAAAMxDf2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3hVRfrA8e+kV1JISIAk1NA7AQEJhCbFiiJWUFGxoK5bVHRZRH/sUnTVRV13LYgiiiBtAZFepUpCpLcQSAKBhIRUUu/8/jghtACB3Nxzk7yf5+Hh5ty557w5htfJnJl3lNYaIYQQVZuD2QEIIYSoOEnmQghRDUgyF0KIakCSuRBCVAOSzIUQohpwMuOiAQEBumHDhmZcWgghqqydO3emaq0Dy3rPlGTesGFDfvvtNzMuLYQQVZZS6vi13pNhFiGEqAYkmQshRDUgyVwIIaoBU8bMy1JYWEhiYiJ5eXlmh2JX3NzcCAkJwdnZ2exQhBB2zG6SeWJiIt7e3jRs2BCllNnh2AWtNWfPniUxMZFGjRqZHY4Qwo7ZzTBLXl4etWvXlkR+CaUUtWvXlt9WhBA3VK5krpQKUkptvM77zkqpxUqpX5VSo241GEnkV5N7IoQojxsOsyil/IBvAM/rNHsZ2Km1nqCU+lkpNVdrnWWtIIUQoqrafCSVrXFnAeiW8AVe4T1p1+s+q1+nPGPmxcBDwKLrtIkCxpa83gBEAGsvbaCUGg2MBggLC7vZOG1i165dAHTo0MHkSIQQVdnmo6mcPJfHf9Yf5ciZbAC81HledfmSXzVgRjLXWmfCDX/d9wSSSl6nAUFlnOdz4HOAiIgIu9wRQ5K5EOJWJKbncirDeLa1Iz6Nqb8cvOz9eS90p7NlL3yjiew9qFJisNZslmzAHcgAvEq+vmXvLN7LvpOZ1oirVKt6tXj77tbXfP/NN99kwYIFAMycOZPIyEi2bdtGbm4ugYGBzJ49GycnJ6Kiorj33nv5+uuv+f33360aoxCiavl2Szxbjp5l2Z7kq977cmQEzYO9CfZxw9nRATaVlDCp36lSYrFWMt8J9AR+AtoDW610XpuZNGkSzZs3B+DJJ59kwoQJREZG8tZbbzFmzBgWLVrEAw88wKlTp1BKSSIXogbbcCiFBTFJLIgxBiSaBHoysHUwPZoEABDi507DgCseMybtBL9G4OFfKTHddDJXSvUFWmmtP7nk8DfAz0qpSKAVsK0iQV2vB21LnTt3BqBdu3bEx8cD4OPjwyuvvGJiVEIIW0jOyGPdwTNcOSb8+YY4jqXmlH498+muRIaXWcjwcknRENbdukFeotzJXGsdVfL3GmDNFe8dV0oNwOidj9daF1szSFtxd3fn7FnjqbPWmu3btzNw4EBiYmIYPHgwAB4eHjg42M30fCFEJcjJL+KJ6ds5eLrsSXnNg7yZ92IPvFzLmUIzT0FmEtTvbMUoL2e1FaBa65PAHGudzwwDBgxg+PDhzJo1C1dXV7y8vIiKiiI4OJi77rrL7PCEEJUoM6+QzPOFnEjL5dEvjMGFFsHezHiq62XtlII63q43twbkZLTxd1VI5tWBv78/q1atAmDChAlERUURFRV1WZt169bZPjAhRKU5X1DMP1cc5MtNxy473qWhH58+2ok6tdwqfpGknaAcoW67ip/rGiSZX8OECRPMDkEIUclW7jvNs99e3Cjn4S6hdGrgR3AtN3o1K8c4eHkl7YSg1uDsbr1zXkGSuRCixkhMz+XImWy2H0tjYUwSJ0vmhjcP8mbpKz1xcqyE52EWCyTFQJv7rX/uS0gyF0LUCMUWzX2f/kpqdkHpsfs71mdM36Y0DvCsvDpIaUchP6NSx8tBkrkQoob4aNUhUrMLGNwmmGd7NSbM34MAL9fKv3DSTuNvSeZCCFEx6TkFfLzmCP6eLrx7bxsCvW2QxC9I2gkuXhDYvFIvIxOmr5CRkUHfvn2JiooqXd4vhKjaXpxlTA3804Bmtk3kAMe3QN0O4OBYqZeRZH6F2NhYevTowbp16xg6dKjZ4QghKkBrTVxKNltKStA+3q2BbQM4sx9O74YWd1b6pexzmGXZWEjebd1zBreFwZOv2+Rf//oXX3/9NefOnWPTpk20bt2auLg4KbYlRBU1bfURPlx1CIChHevbPoDYH8DBCdo+WOmXkp75Jf7whz/w0Ucf8eSTT7Ju3ToCAwOJjIxk/fr1BAUFsWiRUdJdim0JYf82HU5l+V6jmuHHj3Rk0v1tbRuApRh+nwNNB4CXFeesX4N99sxv0IO2JSm2JUTVk51fxONfGUvy+zQP5O729WwfRNxayDoFg6fY5HLSM7+B7du3AxATE0PTpk0BKbYlhD1bvf80bd5eDsD7D7bnqye6mBNI7Gxw84VmlbMZxZUkI93Ajh07iIqK4ty5c1JsSwg7tyM+jae/MZbn929Zh6Ed6+PgYMKm6HmZsH8JtHkAnGwze8Y+h1lMdGVxrT/96U9SbEuIKuD42Rwe/M8WAMbd2ZJnIhubF8y+hVB0Hjo8arNLSjK/Dim2JUTVkFdYTO/31gHw3xGdGdg62NyAdv0AtcMrfdXnpWSYRQhRpSWdO8/Ir4xnW10b+ZufyI+sghObocMjRvFzG5GeuRCiyiq2aO74YD05BcbmZrOf7WZyRMDGD4y/I0bZ9LKSzIUQVdbZ7HxyCop5KCKUV/qHm/Ow87KAjsLxX6HfeHD3s+mlJZkLIaqc/KJi7pq2icNnsgEY0CqI+r6Vt/FDucXMNHYUam+7B58XyJj5JXbt2sWuXbuscq7i4mLuvvtu+vTpw2effWaVcwohDDuPp3P4TDYhfu5MeaAt/VsFmR0SFBfBru8h/A6oVdfml5ee+SUuJPIOHTpU+FxJSUn4+PiwePHiCp9LCHFRYbGldMPlfz3cgc4N/E2OqMThFZB9GjqNMOXydpnMp2yfwoG0A1Y9Zwv/FrzR9Y1rvv/mm2+WlrydOXMmkZGRbNu27ZaKbP344498+OGHxMfHExUVxccff8zOnTv55ptvAHBxcWHOnDn4+PgQFRXFc889x6RJk1ixYgXBwSY/iRfCjuUVFtNzyloA7utQj05hth2Xvq6YmeBZx+iZm0CGWUpMmjSJsWPHMnbsWFavXg1QriJbb7/9dulCo6ioKF588UUeeughZs+ezaBBg1i3bh1t2xoFfsLDw1m7di39+/fnyy+/LL12bGwssbGxksiFuI7E9Fz6f7Ce1Ox8/Dyc+fChDpW31dvNykqGQ8uN6YiOzqaEYJc98+v1oG2pPEW23nnnnVs635IlS0qPjxs3zn5+KIWwM4XFFhZEJ/H6PKMDFdU8kBlPdTU5qivE/gC6GDqONC0E6Zlfwt3dndzcXMAoam/tIltlnQ/Ay8urImELUW3lFhQxasaO0kT+zj2tmW5W4axrsRRD9LcQ1gMCmt64fSWxy565WQYMGMDw4cOZNWsWrq6ueHl5ERUVRXBwsFWKbCUkJNCnTx/c3NyYM2eOFSIWovpKycpnyLSNpGTlA7DqT71oWsfb5KjKsG8RpMVB33GmhqG01ja/aEREhP7tt98uO7Z//35atmxp81iuZcKECVcV3aqIGTNmAPDkk0/e9Gft7d4IUdmOpmTT75/rAfjLHc24o3UwzYLsMJFbLPBZD9AWeHFLpe/zqZTaqbWOKOs96Zlfg7WLbN1KEheiJjqWmsPMLccB+POAZrzUN9zkiK5j3wJI2Q8PfFXpifxG7CqZa63lQeAVzPjNSQgzrD1whq83x7PhUAoAYf4evNzPjhO5xQKr3jGqI7Y2f/N3u0nmbm5unD17ltq1a0tCL6G15uzZs7i5uZkdihCV7tst8UQfT6dTmC+D29TlPjM2YL4Zh1fAueNw5z9N75WDHSXzkJAQEhMTSUlJMTsUu+Lm5kZISIjZYQhRqfYkZbD2YAr3d6zPBw9VfAW2TWz+GGqFQKcnzI4EKGcyV0p9BbQClmqtJ5bxvh8wC6gD7NRaP3ezgTg7O9OoUaOb/ZgQoopLzc7n4c+3AtCjaYDJ0ZRT0k44vgnu+Ltpi4SudMMJ00qp+wFHrXV3oLFSqqxBrBHArJKnrN5KqTKftgohBEBRsYVDp7OIPpHOo19sJTu/iEVjbmdY5yrwW2j6cfhqILjWgk7mLRK6Unl65lHAhUnRK4CewOEr2pwF2iilfIFQIOHKkyilRgOjAcLCwm4xXCFEdfD8d9Gs2n+69OuR3RvQPtTXxIhuwob3wFIIA94Dt1pmR1OqPMncE0gqeZ0GdCqjzSbgTuAVYH9Ju8torT8HPgdjnvmtBCuEqPr2nsxg1f7T1Pd1Z9ydLWkc6EXzYDucQ16Wghz4fQ6E3gYRT5kdzWXKk8yzgQtV370oe2jmbeB5rXWmUupPwFOUJG4hRM22JymD+LM5pV9P+eUA3q5O/PyHSHzc7WO8udwOLYfifOj1mtmRXKU8yXwnxtDKVqA9cLCMNn5AW6XUVuA2YJXVIhRCVFlZeYXc9fGmq44/eltY1UvkAPsWGmVum/Q1O5KrlCeZLwQ2KqXqAYOBh5VSE7XWlxYimAR8DTQAtgA/WD1SIUSV8d3W40QfT2dR7EkA3hvWjg6XjIk3qO1pVmi3riAHDq2Ajo/ZxbzyK90wmZcMnUQBA4CpWutkIPaKNtuB1pUSoRCiSll/KIVxC/cAEOrvTvMgb4Z1Dqn6iwEPLYei83ax2rMs5ZpnrrVO5+KMFiGEKNOCmET++GMsAV6uLHixB6H+HmaHZD17FxhDLGHdzY6kTHazAlQIUXWdyjjPH2bvYteJczQL8uKzxztXr0Senw2HV0LHx+1yiAUkmQshKkhrzb/XHuW3+DSGdgxh3J0t8fN0MTss6zp8YYjlPrMjuSZJ5kKIW7bpcCoTl+7jQHIWUc0D+efw9maHVDn2LgSvILsdYgFJ5kKIckrJymftwTNQsuQv8dx5pq02FoO3qluLife1MTG6SpSfbVRI7DjCbodYQJK5EOIGCoosxCaeY8bmeJb+fuqq9+e/2INOYX4mRGYjh5dDUZ7dzmK5QJK5EKJMyRl5fLExjnnRiZzLLQQgvI4XM0Z1LW3j7eZELbcquPjnZuxdUDLE0s3sSK5LkrkQ4irrD6XwxPTtADQP8qZtfR9ejGpKowBPgn1q0GYpF2axdBpp10MsIMlcCHGJ8wXFPPblVqJPnAPgP493YlCbuiZHZaJDvxhDLK3sdxbLBZLMhRCl/rvhaGkiX/BiDzpW57Hw8ti3ELyC7X6IBSSZCyFKnM3O56NVxuyU2PF34ONRzcfCb6QKDbFAOXYaEkLUDA+VbN32f/e2lkQOF4dY7HwWywWSzIUQ5BYUceRMNk3rePHobQ3MDsc+7JlvDLGE2v8QC8gwixA1msWiefqbHaw9mALA6MjGODpU8eqG1pCbZiwUuu05cKgafV5J5kLUYF9sjGPtwRSimgfSpp4Pw7uEmh2Sfdi30Njns91wsyMpN0nmQtRQ83YmMmmZsYXbFyMjcHasGj3QSldcBFs+haC2ENzO7GjKTZK5EDWQxaJLN5B4f3h7SeSX+n02nD0CD82CKrShhiRzIWoArTWbj54lK6+QnPxiPl5zmPOFxUwd1o6BrYPNDs9+FOXDuilQryO0uNPsaG6KJHMhaoCvNh1j4tL9pV8rBcM6h3BHqyATo7JD0d9Cxgm4+8Mq1SsHSeZCVHv/XX+UScsO0LNpAH+9syUAgd6uBHi5mhyZnSnIhQ3vQ1gPaNLP7GhumiRzIaqx1Ox8Ji07AMDbd7ciPMjb5Ijs2I4vITsZhk2vcr1ykEVDQlRrH5dsHrFwzO2SyK+n8Dxs+hCa9IWGt5sdzS2RZC5ENZWdX8Q3W47TKcyXdvV9zA7Hvh1dA+fToPtLZkdyy2SYRYhqJjE9l0/WHGFL3FkAXukXjoOs6ry+/YvBzRca9TI7klsmPXMhqplxC/cwPyYJF0cHJt7Xht7NAs0Oyb4VF8LBn6H5EHCsugXGpGcuRDWQllPAuoNnSM7MY93BFN4a0oLRvZqYHVbVELce8jKg5V1mR1IhksyFqOKKLZqXvo9m81FjWKVJoCcjuzc0N6iqZPmb4O4PTfubHUmFSDIXogrTWnPPJ5vYezKT5kHefDEygjq1XHFztv/NFOxCUjSkHjIefDpV7Xn3ksyFqMIS08+z92QmAF89GUGIn4fJEVUx278AZ0/o/YbZkVSYPAAVooraeTydyKlrAVjyck9J5DcreTfEfg9tHwC3WmZHU2HSMxeiCklMz+Uvc2NJzynk4OksAO7rUI/W9ap+MrK56G+Nv3v+0dw4rKRcyVwp9RXQCliqtZ54nXb/BpZprRdbKT4hRInCYgtPTN9O0rnzRDWrQ6MAT25vWpsR8rDz5hUVwO6foPX94N/Y7Gis4obJXCl1P+Cote6ulJqulArXWh8uo10kECyJXAjryi8q5rN1R0lIO8/RlBye7NGQCfe0Njusqm3LJ8aKzw6Pmh2J1ZSnZx4FzCl5vQLoCVyWzJVSzsAXwM9KqXu11ouuPIlSajQwGiAsLKwCIQtRMyzalcS4BXvIyi8qPVbXx40X+8j88QrJPgMb/wl+jaBxH7OjsZryJHNPIKnkdRrQqYw2I4F9wFTgZaVUmNb640sbaK0/Bz4HiIiI0LccsRA1wIq9ybz64y46hfnRvXFtArxceKJHQ1QVrOZnd9ZNhqI8eOwncKw+jw3L851kA+4lr70oewZMR+BzrXWyUuo74O/Ax2W0E0JcQ35RMUt/P8X/Yk+yOzGD5kHezHrmNpkzbk0ph2DnDIgYBQFNzY7GqsqTzHdiDK1sBdoDB8tocwS48BQhAjhuleiEqCEOJmcxZNpGii3GL61dG/oz7q6WksitbfU74OwBUWPNjsTqypPMFwIblVL1gMHAw0qpiVrrcZe0+QqYrpR6GHAGhlk/VCGqp3k7E/nz3FgAbmvkz9/uakUbKVlrfSmH4MAS6D0WPAPMjsbqbpjMtdaZSqkoYAAwVWudDMRe0SYLeLBSIhSiGkvJyuedxXtxcXJg9uhudArzMzuk6mvbf8DRFbo8Y3YklaJco/9a63QuzmgRQlhBUbGFEV9tIzOviMn3t5VEXplO/Q4x30GbB8CrepYEluX8Qphk0rIDHEjO4sHOIQyPCDU7nOrt57+Aux8MeNfsSCqNJHMhbCwjt5AvN8bxw/YTtA/15f/uayM7AVWmlIOQsA26j6m2vXKQ2ixC2FRRsYWXfohm4+FUXJwcmPJAW5mxUtlivgMHJ2j/sNmRVCpJ5kJUEq01b87fTVxKTumxuNRsUrMLaBfiw0/P98DFSX45rlTFhRD7AzQbBF51zI6mUslPkhCVZP+pLGbvSCAzrxBHB4WjgyK8jjfDI0KY9cxtksht4fAKyEmBjiPMjqTSSc9ciEry6bojOCiY9cxt1Paq2rvYVEkWC/w6DTzrVPkt4cpDkrkQVnQwOYsDycbOP6v3n6a+n7skcrOsnwwJW+G+z6pVDZZrqf7foRA2cPh0Fu8u2cfGw6mXHf9j/2YmRVTDnU+H9VMhqC20f8TsaGxCkrkQFXQwOYvHv9pGsUXTu1kgg9sE07WRP86ODoT4ud/4BML6dnwFaBgyFWpIpUlJ5kJUwJajZ3nki624ODmw9OWehAd5mx2SyDxp1CtvcRc06GF2NDYjyVyIW5CWU8DaA2dYuMso9f/FyAhJ5PagKB+W/BEsxTDw72ZHY1OSzIW4SecLinn0i60cSDY2VB7WOYTezarvysIqIy8D/tXB2A5u0GTwa2h2RDYlyVyIm6C15m+L9nDwdBafPtqJdiE+1PVxMzssAbD2H0Yi7zYGur1gdjQ2J8lciJvw444EftqZyCv9wrmzXV2zwxEXZKfA9s+h3cMw6B9mR2MKSeZClENKVj6zth3no1WHiQwP4A/9ws0OSVzq149AW6Db82ZHYhpJ5kLcQHJGHpFT11BYrPFwceS9Ye1xlCqH9iM/G2JmQpN+UK+j2dGYRpK5EJfQWpOcmUdRsaag2ML0TcdYticZB6V4pV9TRvdqjJer/LOxK7E/GA8/q+G+njdDfiqFKPHTzkTm/pbAtmNppcccHRQdQ335212taB/qa2J0FVdsKcbR4frldk9mn2Tl8ZU0rNWQFcdX8FjLxwj3DcfZ0dlGUd4krWHrZ1C/M4R0MTsaU0kyFwJISMvlLyWbKo/o1oB2IcaGyh1Cfav8/PHC4kK2nNrCa+tf46/d/so9Te4pfS+7IJsRy0YQFRrF7pTdbEvedtln/3f0fwxuOJipvafaOuzyObYB0o7CnR/UmJWe1yLJXNRoeYXFvPR9DKv2nwbg66e60Kd59al7XWQpYvD8wZzONb6/cZvGUWwpZmj4UKZFT+OL3V8AcOTckcs+19inMXEZcQAsi19GZmEmH/f52P566Bv/afzdpK+5cdgBSeaiRvvX6sOs2n+au9rVZUS3BtzWuLbZIVnVX9b/pTSRfxj1IXMPzWX85vGM3zz+snZ3Nr6TpXFLeaH9CwxtOpS6Xsa0y4LiAjp/15lfk36ly6wudK/XnTe7vklYrbBrXrPIUsTk7ZPpFdKLXiG9Ku+bS4qGY+uhzzjwb1R516kilNba5heNiIjQv/32m82vKwTAqn2nWX8ohWKt+XFHAsM6hTBlWDuzw7Ka31N+59Ndn1LPqx4/HfoJgJXDVhLsGUx+cT5/WPMHfj35KwD3Nb2P17u8jreLN1prVBlDFXEZcby34z02JW0CwMPJg3HdxnF3k7tL2xRZilhzYg2J2Yl8uPPD0uPPtXuOLSe3EBUaxTNtnynz/BfkFuaSXZhNHY9y/mb04+PGMMure8CtVvk+U8UppXZqrSPKfE+SuagptNa8/tPvzN2ZCIC/pwsNansw48mu+HjY2fDBTUjMSmRH8g5OZJ3gsZaP8eKqF9mftr/0/b/3/Ptl4+TFlmJSzqcQlxFHt7rdcFDl2/Ho+/3fcyDtAMczjxN9JprBjQZzvvA8Tf2aciDtQGmyv6B17dbsPbu39OvhzYbz1m1v4ejgyInME8RnxhNZP7I0wU/cOpGf435m/r3zCfYMvn4wacdgWkfo9RfoO65c8VcHksxFjZeeU8DkZQf48bcE7u9Unwn3tKaWm/0n8LJmoOQV5eHmZJQQyCnModv33Urfc3FwocBSwLs93sXdyZ0GtRrQsnZLq8ZUZCnii9+/4N+x/77qvVFtRvFA+APU86qHg3Jg0rZJzD44m/vD72f+4fkMaDCAt257iz5z+gDwbNtnebbds4z/dTy/xP8CgL+bP18P/Jqlx5aSnJPM4y0fv/p7WP0ubPrQ6JX71Lfq92fPJJmLGu+Zb3awav8ZHuwcwtRh7a77674ZCi2FpOamEuwZjEajUMzcN5OPoj+iqW9TTuWc4qUOLzFx20QA2tRuQ5Eu4kDagdJzdAjsgI+rDxZt4ZN+n5S7x32rYs7E8OXuL2kf2J69qXt5rv1ztKrd6prtv9n7De//9v51z+nr6su5/HNXHd81YtfF/6mdS4CP2hibND/6Y4W+h6pGkrmo0ebtTOTPc2N5KCKUyQ+0tbtE/vzK50vHsC9o4d+CA2kHaBfQjuNZx8nIz7js/TDvME5knQDAQTnw68O/4uXiZbOYb9X/jv6Pv276KwCxI2P5cOeHzNg7A4CxXcfSJ7QP7259l1+Tfr3qs58P+Jzu9brDojEQ8x08uRQa9rRl+KaTZC5qrFnbjjN+0V78PFxY/mqk3e3HuTRuKWM3XnvlYvSIaBSK9Lx0fj72M5/u+pSZg2fSzK8Zh88dxt3JnVDvUBtGXHEH0w5S2702Ae4BAGQVZJGel37ZDJkdyTs4mX2Su5vcTftv25cenxAYyQPbZ0HX0TDkPZvHbjZJ5qJGOnE2l17vrcXZURH9twF429kY+YbEDYxZPQaA6QOn0yXYWMGYX5zPgsML6N+gf2nCq8nS89KZe2guH8d8DMATOYU8/9ASvGo3NTky25NkLmoUi0Xz8Zoj/BSdQELaeX55NZIWweZMXZu5byb5xfm0DWhL69qt8XT2ZE3CGuYdmkdcRhxJ2UmXJXJxbS9O78RGx8LLjq0dvrZG/Q/veslcFg2JaqWw2MIbP/3O/JgkOoT6MjqysWmJ/ETmCabuuLgMXqHQXN55+kfPf0giL4+sZP594ijPNm7FVp1derjPnD7sfmK3iYHZD0nmolo4k5nHnN8SWLYnmb0nM/nLHc0Y06epzR52Lj66mLc2vQXAgAYDGNVmFM+tfA6AOXfNIS0vjd9Tf2d3ym42Jm1kVJtRtAtsR5/QPjaJr0qzFBv7eioHvhg8gwL/Rry69lU2Jm0EYM2JNdxe/3ZcHe3reYitlWuYRSn1FdAKWKq1nniddkHAL1rr6xYVlmEWYU074tN48D9bAAj1d2fsoJY22wXo7PmzRM2Jum4b6TlW0K//gpXjYfB7cNtowFgAtiFxAy+teQmAOu51WD18tZlR2kSFhlmUUvcDjlrr7kqp6UqpcK314Ws0fx9wr0CsQtyU05l5pYl80v1tebhLqE164/+N/S9L4paQX5xfemz5A8txc3LDUTky//B8fjr0E2O71uwa2xWWmwYb/gnhA0sTOYBSit6hvQnxCiExO5Ez58+QU5iDp7OnicGaqzzDLFHAnJLXK4CewFXJXCnVF8gBkss6iVJqNDAaICzs2kV6hCivtJwC3l9+EIA/DWjGI11t83O1LmEdn+z6hCCPIE7nnsZRObLjsR2XVRR8qs1TPNXmKZvEU20VF8Kil6AgC/pPKLPJvHvmcdv3twHQ7ftuDG06lECPQPqG9aV17da2i9UO3HCYpWSIZZrWOlYpdQfQSWs9+Yo2LsByYCiwUGsddb1zyjCLqKhpqw/zwcpDADzZoyET7rHNP9yErAQeWvIQIV4hzBwyk6SsJPzd/PF1q9obV9idwjyY9zQcWAIRT8NdH1yzqdaadt9eXSht7t1zaeHfojKjtLmKzmbJ5uLQiRdQ1hrhscC/tdbn7G11nahejqXm8OKsaPafygTgH0Pbcnf7yh0fL7YUM37zeOIz4vk99XcAPoj6AFdHVxr7Nq7Ua9dIBbkwYwicjIFuY2Dg36/bXCnFymErGc6DSoQAACAASURBVDRvEMW6uPT4g4sfBGDTw5vwcfWp1JDtQXmS+U6MoZWtQHvgYBlt+gN9lVJjgA5KqS+11s9YL0xR0yWk5fLMN79xMuM8jg6Kl/o05dnIxlatdvjRzo+o61mX2u61CfQIpK5nXZYdW8bK4yuJTYktbfe3bn8jxDvEatcVl9AaFr4AJ3fBXR9C56fKtYNQsGcwMSNiACO5j1k9hg2JGwDoObsnS4YuoUGtBpUautnKk8wXAhuVUvWAwcDDSqmJWuvSupNa69IK9EqpdZLIhbV9uvYIB09n0bWhP5MfaEvjQOvUIcksyGTUL6M4mF5WH+Wika1GMrjRYLTWtA1sa5VrizLsnAH7FkL/dyBi1E199NJRgU/7fcqp7FPcMe8OAOYdmsefIv5kzUjtTnmnJvoBA4ANWusyH3DeDBkzFzcjv6iYbv9YTfNgb2aP7l7h8xVbitFonBycmLB5AguOLMCiLQB0r9udOxvfia+rLwlZCTTza0a7wHalJWdFJTq0HOaMhLBu8PgCcKh41ceEzAQeWvoQWQVZpcfWPLiGQI/ACp/bDBVeAaq1TufijBYhbCY1O5/xi/aQnlvIi1EVr8WxIn4Ff17/Z5wcnAj1DuVYxjGeav0UL3d8GScHJ7urqFgjZCTC2kmw6zvwqA33/ccqiRwgtFYor3Z6lf/b+n+lx/rO7UvsyNhKLxFsa9XruxHVyucbjhIxcRU/706maR0veja99RocaXlpfLv3W17f8DoAXYO70rBWQ/qG9uWFDi/g7OgsidwMZ4/CVwONRN58CIzZDrWs+0D70u3tLlgat9Sq17AHspxf2JUzmXnkFRpDHnN+S6SejxtDO9XnlX7hODjcerJ9c+ObbD65mYa1GjLrzlnUcqkZe0batVO/w7f3wvk0Yx55zz9WymXcndzZ9ug2nBycsGgLT/zyBNNipjGgwYBqNXwmyVzYBa01H648xLQ1Ry47/kJUE14beOtzhbXWvLPlHTaf3EyQRxBf3PGFJHKz5WXCnnmw4m/g7gtP/Qx1rLu13ZU8nD1KX/8l4i+MWj6KLrO6sP2x7bg7VY9F65LMhSnOZOaxICaJ2TsSOJaaU3r8vg71iAw3Hk45OECf5uXcqb0MhZZCOs3sBBj7Ss67Z16NmG9s184cgJn3QdYp8Aoydgvys+2UwS7BXfBx9SEjP4PVJ1ZzV+O7bHr9yiLJXNjc7O0nGLdwD0UWTfMgb17p2xSUoq6PGw9FhN7ScIpFW8gpzMHbxRutNV/u/rJ0t3gnByfWDl9b7R54VTlJO+G7YeDoDA9+A80GgrM5veJ1w9cxZP4Qvt7zNYMbDr5q0+yqSJK5qHQ5+UWcysjjxx0nOJCcxd6TmRRZND88241ujf0r/OAx7lwc9y66F4AZg2YwY+8M1iWsw9nBmWZ+zZh791xJ5GY7tgF+eMSYrTJyIfibu3LWycGJfmH9+G7/d4zfPJ6/97z+KtOqQHYaEpVCa02RRZNfZKHN28sve69jmC/P9GxslTK1yTnJDPhpwFXHn2//PM+0fQaFwsXRpcLXEbdIa2Mh0LI3wL8RjFgAteqZHRVw+c9OzIgYnBzsv28rOw0Jm0rLKeDhz7dw6PTFHWFub1qbR7qGERkeiI97xZbgFxQXsPXUVv539H/kFxklaO8Pv59+Yf1YGreUvmF9GdhwYIWuIaxk8zSjFnnd9jBiIXj4mx1RqWDPYKb1mcYra1+h48yOVb7uvCRzYRUFRRZ+PZpKfqGFb7fEE382l1f6hePiqAiq5caDERXfQT49L50j547w6tpXySzILD0+otUIXu9izB/vFdLrWh8XtlRcCMvfgu2fQ1gPY2jFyf52AooMiSx9veDwAoaGDzUxmoqRZC4qJLegiNnbE5ixOZ4Tabmlx6c+0I7hXSqewC+Yc3BO6So+FwcXwv3CGd5sOHU86tCjXg+rXUdYwb5F8Os0SPoNerwM/SaAo32mGicHJ+bcNYfhS4azLXmbJHNRM83cEs+Hqw6TllNAeB0v/vlge1rWrYW3mxOh/h43/Hx5aa2ZfXA2AE+1NjZ98HPzs9r5hRUUF8HeBRD9DcQbe3Ny33+gwyPmxlUOLWu3pE9oH3Yk70BrXWVXAksyF7fkTGYeb/9vLyF+Hkx9oB19W9Sp0ArNsvx9699LkzjAq51e5em2T1v1GsIK4tbB9w9D0XljlkrHx2HIP8G56qyu7Fa3G2sT1vLHdX/koz4fmR3OLZFkLm7awpgk3v7fXiwaZjzVpULlaPOK8rBoCx7OHmxI3MC06GnUdq9NE98mLI5bfFnb+5reV9HQhTXlZcCaica4uGcg9HwVIv9it0Mq1zOgwQAmbZ/E6hOrOV90vkquCq16d12YZn50It9uOc6JtFxcnRx4557Wt5TID6Yd5D+x/yGnMIc9qXvIKswiyCOItLw06nnVIz0vnTkH55BfnM+rnV6lS3AXTueeprZ77Ur4rsQtObEV5j0DGQnGBhID/w4uVXcz5UCPQGYMmsGTvzzJnINzeKL1E2aHdNMkmYtySc8p4E9zjN12IsMDGNY5hHs71L9m+7Pnz+Lk4ISPqw+z9s/idO5pfo77GU9nTzILMkk9nwpA75DetA1oS3xmPI7Kkde6vIaPqw/FlmLO5J6hjkedarE6r9qwFMOG92H9ZPANM5bjN+xpdlRW0TmoM7fVvY3pe6bzYLMHL6vnUhVIMhfXtfdkBiv2nub3xHMAfP1kF/q0uH69lPiMeB5d+ihZhVl4OnuSU5hz2fvODs50Ce7Cx30/xtO57N6co4Mjdb0qd29PcZMyTxq98eO/QtvhcOc/wa16FS0b02EMI5eN5MXVL/L1wK+r1MNQSebimlbvP83LP8SQW2Bskvtc78ZlJvLsgmzWJ65n88nNRrnRU9twdHBkTIcx7Endw6akTcy7Zx6BHoF4O3tTZCmSjSCqmnMnYMadkHMWhv4X2j9sdkSVomOdjgDsPL2Ttza9xaTISSZHVH6ynF9cJSe/iPGL9jIvOhF3Z0fmvdCDVvWu7oEtPrqY6Xumc+ScUbbWy9mrdEn0e73fo1vdbjaNW1SSlENGpcOCbGMVZ/1OZkdUqY6eO8p9i4yH7Rsf2oivm6/JEV10veX8kszFZfafyuSl76OJS81hTFRTnu7ZCD/Pq2ubnMs7R+SPxuq5iKAIBjQYwOBGg/Fz86vSc3XFFQrz4LPuxtZuo5ZX+0R+wdxDc3l3y7sAdrXMX2qziBvSWjNr2wneXbIPH3dnZj19Gz3K2KatoLiAeYfnMXX7VAC+H/L9VbvVSyKvJgrPw+JXIS2uRvTILzUsfFhpMo/PiKehT0NzAyoHSeaCzLxC3py3m6W7TxEZHsCHD3UgwOvqOho9fuhBVkEWDsoBi7bwYocXr0rkopooLjS2dEvYZjzsbNLH7IhsSinF1F5TeX3D66w4voLR7UabHdINSTKv4WITzvHSD9GcPJfHG4Na8Fyvxpet5MwryuNs3lmOnjtKVkEWAMObDadPaB961JeaKNXSmQOw4Dk4tQv6/BV6/snsiEwxuNFgvt//PSviJZkLO6a15qtNx5jyywHqeLsx57ludG5weXnSM7lneGHVCxxKPwRAPc96LB66WOqDV2cJO4weubMbDP8WWt1rdkSmGthwIFN2TKkSQy2SzGugzUdTmfLLQWITznFHqyCmDmuHr8fFBG3RFr7a/RXTYqYBRolZX1dfutftLom8Oks7Bt/db+wG9PQKqCXz/Ps36M+UHVOqxFCLJPMaZFfCOcYt3M2epEzq+rjx3rB2DOscctkDy9Tzqby58U22ntpKz/o9uaPBHdzX9D55qFndFeTA9EFQlA/Dv5FEXiLYM5iOdTqyPH65JHNhvmKLZtzCPfyw/QRBtVx5++5WPNI1DDfny5fJbzm5hTc3vkl2YTZvd3+bB8IfkCReExTkwncPQHYy3PNxjZq1Uh53NLiDKTumcCzjGI18GpkdzjXJLrfVnNaaCf/byw/bTzA8IoRVf+rNU7c3uiyRF1mKmBY9jedWPoePqw/f3/k9w5oNk0ReE+z6Af5zu1E4695/Q6eRZkdkdwY0MPYJXRG/wuRIrk965tWY1pp/rzvKzK3HGd2rMW8NaXlVm+ScZN7Y8AbRZ6IZ2nQoY7uOrXIFhsQtyE6BZa8ZG0o4usCwr6DNA2ZHZZeCPIPoWKcjK46v4Ln2z5kdzjVJMq+mDiRn8ursXRxIzuKe9vUYO6jFZe8XFhey8vhK/rH9HxQWFzI5cjJ3Nr7TpGiFTcV8B8vGQkEW3PY89BtfpcvX2sLAhgOZvH0ycRlxNPZpbHY4ZZJhlmpGa80ff9zFkH9tJDW7gLeGtOC9B9uVzh0/l3eOBYcXMGTBEN7Y+Ab1POsx5+45kshrih1fwqIxULsxPLsGBk+RRF4O/cP6A/Y91FKunrlS6iugFbBUaz2xjPd9gNmAI5ADPKS1LrBmoOLG9iRl8LdFe4g5cY4eTWoz7ZGOpSs5U8+nMvfQXKbvnk5ecR71POsxut1oRrcbjauj/e2aLqysIAeiv4VfxkKzwcaMFSf5715eQZ5BdKrTiRXHV/B8++fNDqdMN0zmSqn7AUetdXel1HSlVLjW+vAVzR4DPtBar1RKfQYMAv5XCfGKMmitmbn1OBOX7MfdxZE729Xlw+EdcHEyfvHae3YvDy8xSpa28G/B8ObDGdp0aGmFQ1GNZafAb9Nhw1SwFEGLu2DY1+Ak6wVu1h0N77DroZby/GuOAuaUvF4B9AQuS+Za639f8mUgcObKkyilRgOjAcLCwm4hVFGWjPOFvPHT7/yyN5k+zQP55/AO+JdUOYw7F8epnFM8v8roSbza6VVGtRkls1RqgqICiP0BFr9ifN0wEjo8Bm2HgaOzubFVUf3D+jNl+xRWxNtn77w8ydwTSCp5nQZccxKqUqo74Ke13nrle1rrz4HPwSiBe/OhiivFnEjn5R9iSM7I460hLXimp1FX5ZOYT1h2bBknsk6Utv2036f0CullYrSi0hUXQfQ3xk5Ap2Lh7BFwdIX+E4wHnQ7yiKwiLsxqWR6/vMom82zgwlbVXlzjoalSyh/4GJD5TVZWWGwh+ng6sYnniE3IICU7HzREn0gnqJYbc57vTsdQX2btn8WUHVNKP1ffqz6ujq5M7TWV5v7NTfwORKU7GQOLXoLTe8AnFLyC4KHvoGl/cK56O83bqwtDLb/E/8KghoPMDucy5UnmOzGGVrYC7YGDVzZQSrkAc4E3tdbHrRphDZaTX4QGPl5zmP+ujwMgxM+d+r7uKAd4MCKEsYNagmMur6x9hXUJ66jrWZdQ71Be7vgyHep0MPcbEJWvqAB2z4GfXwN3P6M4Vst7QIbSKsWABgOYvH0yr61/jWCPYLv6N1aeZL4Q2KiUqgcMBh5WSk3UWo+7pM3TGMMvf1VK/RX4TGv9o/XDrTneWbyXr3+NL/3ayUGx9a1+V9UZ33VmF69teI3U86m80eUNHmv5mIyJ1wSFebB/MayaAJmJULcDPDYXvK6/2baomDoedWjp35L9afsZsWwE0SOicXawj2cQ5do2TinlBwwANmitkyt6Udk2rmzbj6Xx+k+xZOYVkZZTwN3t69Guvg8AnRr4EhZYzO7U3RRaCgn3DWdd4jqmRU+jrmdd3u/9Pq0DWpv8HYhKd/AX2Dvf+Ds/AwJbQtRYaDZQhlNsJL84nzc2vMHqE6uZ2msqgxsNttm1ZQ/QKmDn8TRGfLWdOt6u9AwPIMjbjZG316WWqyffH/ieydsnl/m5AQ0G8E6Pd/B28bZxxMLm4jcZBbGK8qDdw9D+YWjUCxwcb/xZYVUWbeHO+XeSmJ3I7Ltm07q2bTpSsgeoHdNas+XoWUbP3EmdWq5MGOZN09p+fLP3G3r+OLt0izaAN7q8QZuANrg4unDk3BE8nTzpG9ZXhlWqu+Q9sPUz2PUd+IYZ4+L1OpodVY3moByICo3iu/3f8fCSh4kdGYuDMne2kCRzk03+5QD/XR9HWG13enbdxMvr55S+N6jhIEK9Q8ksyCQiOOKyp+etarcyI1xhCwU5sPgPxmYReRlw9jA4OMHtf4DeY8FFCqHZgz92/iPf7f8OgDUn1tC/QX9T45FkbqKUrHxm/BpPy7redO28kflH5/BIi0do6tuUjPwMnmzzpN08XBE2cnIXLHwBzuwzvm7aH257DlrfD561zY1NXMbF0YVdI3Zx76J7+ce2f9ArpJepO3FJMjfRlxvjKHaJwzVsLfOPHuTxlo/zepfXZdikJjqfbkwv3D3XmGL4+Hxo2s/sqMQNODo48nSbpxm/eTy9f+zNlke3mBaLJHOTpOUUMHPrMWo3XcTRzFM81vIxSeQ1idaQ/DucOQAxM41FPwXZEPlnYzjFzcfsCEU53dX4LsZvHk92YTbni87j7mTOrCJJ5iawWDR/mB1DgVssufoU7/V6j0GN7Gs1mahEib8Z88PjNxpfO7qCbyjcPQ0a3m5qaOLmOTs6M33gdEYtH8XK4yu5p8k9psQhydwEy/cms/HwGYJbbiSwVsPSbalENWSxQFqc0fM+tcsYE0/YajzQHPB/0LAn1GkFzm5mRyoqICIogjDvMOYfni/JvKY4mpLNqz/uIjhsMzkk8Nd2/8BR5glXP+dOwNK/wOEVQMlaDkdXCG4DXUdD79eNsXFRLSilGBo+lH9F/4vjmcdpUKuBzWOQZG5D+UXFvPJDDNrlBDmeS/B19bXp6jFRybSG45th22dwYKlxLPwOaHk31OsAgS2k/Gw1dm+Te/kk5hMWHF7Aq51ftfn1JZnb0PvLD7IvJR6vpp8A8PmAz2WDiKpOa6Nu+IGlkHrI+OPuBz1egS7PGGPhokYI9Agksn4ki44u4qWOL9n837ZkEht5bW4sc3ceJ6zNfNKLYUL3CbSs3dLssMTN0BqSoo1ZJwAno2HrfyA72RgD965nPMRs+6As7KmhhoYPZV3iOjYmbqRPWB+bXluSuQ0kpOUyL3Y/Ic3WkF58hPd6v2d3tZDFNRTlQ9YpYzXmmomQdEVNoUa9jEJXnZ6QzR8EkSGRALyy9hXGdx/Pg80etNm1JZnbwPRNx3AP+5wMxxTua3qfJHJ7ZSmGLZ9CygFIPw7p8ZCZROkDTK9guOtDCCjZ6MMrCAKamhWtsEPODs4MaTSEn4/9zLtb3pVkXp1k5hUyZ+9aHOql0NinMW92fdPskERZTsYYc7/j1oFHAASEQ6NI8GsIPiHg5AZN+oKHv8mBCns3pdcUvF28+fHgjyTnJBPsGWyT60oyr0T5RcU8PWMHFu/1+Dr7MefuObg6ut74g6JyZJ02ls2nxUHaUUg5CJknIT8LEreDcoQ7JkKPl82OVFRxT7R+gh8P/siSuCU80/YZm1xTknklKCy28NPORD5Zc4QzaiVuwQd4vNWLkshtLfMUHPzZeGiZst/ofZeUEwbAo7ZRUtbBCfr8FTo+DrXqmRevqDZCvUPpVKcTi48u5uk2T9ukTIckcytKyylg+d5kPl17hJM58QQ2WIObUywAw5sPNzm6GuDMflg3yaj/rS1w7rjxt5sP+DeBiFEQ1t0Y667TSqoQikp1d5O7eWfLO2xM2kivkF6Vfj3ZachKNh5O4dlvt5Ov0wluuJ4c5214OnvSo14PRrcbTXP/5maHWL2dT4f/9ILcVGNxTu0m4NcIGkdBWDfZjUfYXGZBJrf/YNTa2f3EbqucU3YaqmSr9p1mzE+LcWs8HWeHLAocXHii5RM83eZpfN18zQ6vespJhb0LYM98yEkxxsC1BZ76BRp0Nzs6IajlUotgz2CSc5LZdWYXHep0qNTrSc/8Fm2NO8ukX2I4eS6fc0UJeIRNx9UJhjUfxshWI232BLtGyE0zKgwm7zGmDeamwfFNxntBbcG/IdRpbcw2CbvN1FCFuFTq+VT6zDEWD1mjdy49cytKSMtl4s+7WJs8B9eADShPCx44EOwZxPRBX1Lfq77ZIVYveZkwfaCxTB6gdlNjF/puL0Kre40hFCHsVIB7QOlrrXWlPgiVZF5OOflFfLr2MNNj5+NUexmugRn0D7uDhj5hnD1/ljEdxhDkGWR2mFWXxWKMexedh7NHL9Y52bsQcs/CgzMgtBvUqmt2pELclPHdx/PulndJzEoktFbl1eqRZH4DFotmfkwSk1evINd7Hs7BJ2jq04K/dZ9Gp6BOZodXNeWmwdE1xp/sM5CVDGePGIn8Ui5e0KCHsfuO9MBFFdU2oC0Au1N3SzK3NYtFE5t4jl/2JrD46ArS1Racgg7h7+LPnyPe5d6m9+KgpA5HuWSfgRNbjIU6WachcQck7QS0UV3Qr6HR227cG3xCwckV/BtBQDPwrguyjZ6o4pr4NsHN0Y3dqbsZ0nhIpV1HkvklDpzKZPK6n4k9vZ/zjodx8jqI8i7ACXiq9ShGt3sWLxcvs8O0H5bii73q1ENGjzs31RgWKS405n2fPXyxvbMH1GkJvd+A8AFQr6NMGRTVnrODMy1rt2RP6p5KvU6NTuaHU0/xza5fiD69i+L8ABIKNuPongh+4O/kT7+we2hbpyWR9SOp61UDx2qLi4wpf3kZRsGpzJMX/6TFwalYSotQXeDmY6ysdHAC/8bGqsoGt0OdFuDqbcq3IYTZ2gS0Yc7BORRaCnF2qJwNSmpUMrdYLCw7HM1P+1awO30reQ7xKKXB4gyOhfh6BfFkmz/TM7QrLfxbVP2hlAs7wGckllQCjIf8bEg/BoXnja8tRdf+vKWYq5K1k7sxLOIbBhFPGUMjgS2MZO0TKjvpCFGGdgHtmLlvJkfSj1TaPgbVPpmfzc1i5q7VrIxfS0LeTrRjBgCuNKBzrYd4oMUAhjTvTFJOIvW86lXa/zVtKicVYmYawxy//3jxuLOn0XP2DQXPOlA/AryvNwNHGSspPQKMmiW16hnj3DKOLcRNaRPQBjAegkoyvwm/JR7hu93L2XH6VzL0fpRDEdriSoBjW3rU68nIDgNpERhy2WfM2IC13LQ2KvwVFwAaslPgzF5jvPpKmSdh38KLX7d7CDo8agx51AqRDRSEMEF9r/r4ufqxJ3VPpdVpqhbJPK+wgHn7NvO/Q6s4lLmDIqeTADgUBxDuOYAhTfrxSLveeLm6mRxpiaICyEgwxqG1Nl7nZZS8l2esdDx72NjlJi8Dsk+XfR4XL7hyKMjRBdo/Cp2fgJCukryFsANKKdoEtGF3qnVqtJSlyibzE+dSmBGznA2JGzhdGAuOuWjtgJdDON38R/FImzvo2aAlDrZMZpZiOHeCy8aZU48YU/HyMuDIKmOsOicFivOvfR6fMGMM2tnDGBZx8wHvYPAt+e3B3c+YFSIbJQhRZbQNaMumpE3kFObg6exp9fOXK5krpb4CWgFLtdYTb7VNRcWcPMZnv/3ErtQt5DocMR5eFntS16UTfUKjeKLjAOrVuoUEp7Uxta4g20i2aXHGMbhkeKOsz1mMcenTe4xEbim8dlvlYJRdDb0NnFwgsKWx5ZijizH740IdbeUIrjL9UYjqpk1AGzSafWf30SW4i9XPf8NkrpS6H3DUWndXSk1XSoVrrQ/fbBtr2LbzW7ak/0TjYuhxXtHH4kwnBwtO56MhIxr2fHBrJ87PKtnr8Trcr/E/Cf9Gxri0k6vxYNC/iVE75AJHF2jSx+hNCyFqrEsfgpqSzIEoYE7J6xVAT+DKRH3DNkqp0cBogLCwsFsK9pEWvRm0bQ8Nfdxv3PhmODgZu6x7l8wl9w0D11rGa6WMjXxl7FkIUQF+bn4MaTSEOh51KuX85UnmnsCFbmsaUFZBkhu20Vp/DnwORgncm44U8AmPwic86lY+KoQQppvSa0qlnbs83c1s4EJX2OsanylPGyGEEJWkPEl3J8awCUB7IP4W2wghhKgk5RlmWQhsVErVAwYDDyulJmqtx12njdQrFUIIG7phz1xrnYnxgHMr0EdrHXtFIi+rTYb1QxVCCHEt5ZpnrrVO5+JslVtuI4QQonLIg0ohhKgGJJkLIUQ1IMlcCCGqAaX1La3fqdhFlUoBjtv8wuYIAFLNDsJOyL24SO7F5eR+XHS9e9FAax1Y1humJPOaRCn1m9Y6wuw47IHci4vkXlxO7sdFt3ovZJhFCCGqAUnmQghRDUgyr3yfmx2AHZF7cZHci8vJ/bjolu6FjJkLIUQ1ID1zIYSoBiSZCyFENSDJ3EqUUl8ppbYopcZd430/pdTPSqnflFL/tXV8tnSje3FJu38rpe62VVxmuIl7EaSUirFVXGYox78RH6XUMqXUCqXUAqWUi61jtKXy/GyU9+cHJJlbxaV7oAKNlVLhZTQbAcwqmT/qrZSqlnNqy3kvUEpFAsFa68U2DdCGynsvSrzPxQ1eqp1y3ovHgA+01ncAycAgW8ZoS+W5Hzf58yPJ3EqiuHoP1CudBdoopXyBUCDBNqHZXBQ3uBdKKWfgCyBeKXWv7UKzuShu/HOBUqovkIORwKqrKG5wL7TW/9Zaryz5MhA4Y5vQTBHFjX82ytOmlCRz67hyD9SgMtpsAhoArwD7S9pVR+W5FyOBfcBUoKtS6mUbxWZrN7wXJUMJfwPG2jAuM5Tn5wIApVR3wE9rvdUWgZmkPPej3PcMJJlbS3n2QH0beF5r/S5wAHjKRrHZWnnuRUfgc611MvAd0MdGsdlaee7FWODfWutzNovKHOXaJ1gp5Q98DIyyUVxmsfreypLMraM8e6D6AW2VUo7AbUB1neBfnntxBGhc8jqC6lt0rTz3oj8wRim1DuiglPrSNqHZ3A3vRclvKXOBN7XW1fVn4gLr762stZY/FfwD1AJigQ8whlDaAxOvaNMV2Ivxf9uVgJfZcZt4L7wx/tFuALYA9c2O26x7cUX7dWbHbPLPxQtAOrCu5M9DZsdt8v24so3P9c4pK0CtRCnlBwwANmhj+KDGkntxkdyLi+ReXK489+Nm7pkkN4FyEAAAAfVJREFUcyGEqAZkzFwIIaoBSeZCCFENSDIXQohqQJK5qLKUUh2UUh1KXk9QSkVZ8dw3db6SqYUVPo8Qt0qSuajKOpT8EaLGk2QuqiSl1CSM1ZNjlVKrSw4PUEptUErtUkoFl7Rbp5R6Tym1vORrD6XUTyXtPi055q6UWlJybIFSyqms8ymlXJVSPyil1iulZl2rql9JhcxVSqm1GPU1hKh0ksxFlaS1fhOYDEzWWvcrOdxUa90LmA/0LTnWDdiitR5Y8vVoYE9Ju7pKqXZAK8BScuxrjKXTZZ3v2ZLP9gYOc+0l56OBJVrrPkChdb5jIa5PkrmoTr4t+fsEcKHXvEdrPf+SNs2BoSVj3I2B+kA0sEcptQIYCORe43ytgG0lx7YCLa8RRyOMlXsAv93qNyPEzZBkLqqy84BHyWuFUUb2StlXfH0Q+EhrHQWMw0jU7YFftVFH2w/+v507NkEgCMIo/KYBYxswuhoswFSbMLIgNbg+zA7ODi4XKzG4AUFYFATB4X3JbjC7bPQzLOyyztrX/SbmTp8cp8a57kCXc+/09ROGuf7ZBdhGxMgzgN85AZuIGIA987/yN+AQEVdgSbubPgNdrl0BfaPuCOyy+198eC7pKz7nl6QC7MwlqQDDXJIKMMwlqQDDXJIKMMwlqQDDXJIKeACZOVT5c66zkAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(thres[1:], tpr[1:])\n",
    "plt.plot(thres[1:], fpr[1:])\n",
    "plt.plot(thres[1:], tpr[1:] - fpr[1:])\n",
    "plt.xlabel('threshold')\n",
    "plt.legend(['tpr', 'fpr', 'tpr-fpr'])\n",
    "plt.gca().invert_xaxis() \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4744656418256471"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(tpr - fpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>阈值</th>\n",
       "      <th>假警报率</th>\n",
       "      <th>命中率</th>\n",
       "      <th>TPR-FPR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.930369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.930369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002874</td>\n",
       "      <td>0.002874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.867342</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.034483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.864187</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.032598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.857303</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.040230</td>\n",
       "      <td>0.038345</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         阈值      假警报率       命中率   TPR-FPR\n",
       "0  1.930369  0.000000  0.000000  0.000000\n",
       "1  0.930369  0.000000  0.002874  0.002874\n",
       "2  0.867342  0.000000  0.034483  0.034483\n",
       "3  0.864187  0.001885  0.034483  0.032598\n",
       "4  0.857303  0.001885  0.040230  0.038345"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# KS值对应的阈值\n",
    "a['TPR-FPR'] = a['命中率'] - a['假警报率']\n",
    "a.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4744656418256471"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 另外一种获取KS值的方式\n",
    "max(a['TPR-FPR'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>阈值</th>\n",
       "      <th>假警报率</th>\n",
       "      <th>命中率</th>\n",
       "      <th>TPR-FPR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>0.27769</td>\n",
       "      <td>0.255419</td>\n",
       "      <td>0.729885</td>\n",
       "      <td>0.474466</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          阈值      假警报率       命中率   TPR-FPR\n",
       "224  0.27769  0.255419  0.729885  0.474466"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 获取KS值对应的阈值等信息\n",
    "a[a['TPR-FPR'] == max(a['TPR-FPR'])]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
